setwd("")

library(data.table)
library(ffbase)
library(ff)
### Might want to set the below
## Controls
# options("fftempdir"="D:/R_Tmp")



years <- seq(2000, 2016, by = 1)

df <-
  fread(paste("FollowTheMoneyDownload", years[1], ".csv", sep = ""), stringsAsFactors = T)


df[,which(grepl("token", names(df))):=NULL]

df <- na.omit(df, cols=c("Candidate","Contributor","Total_$"))


df <- as.ffdf(df)


for (jj in 2:length(years)) {
  tmp.df <-
    fread(paste("FollowTheMoneyDownload", years[jj], ".csv", sep = ""), stringsAsFactors = T)


  tmp.df[,which(grepl("token", names(tmp.df))):=NULL]

  tmp.df <- na.omit(tmp.df, cols=c("Candidate","Contributor","Total_$"))



  tmp.df <- as.ffdf(tmp.df)
  df <- ffdfrbind.fill(df, tmp.df)

  rm(list = c("tmp.df"))
  gc()
}
write.csv(df, "temp.csv", row.names=F)
df <- read.csv.ffdf(file="temp.csv", header=TRUE, VERBOSE=TRUE, first.rows=10000,
                    next.rows=5000000, colClasses=NA)


start.df <- read.csv("state_list.csv", stringsAsFactors = F)

for(ii in 1:nrow(start.df)){
  if(start.df$leg[ii]=="House"){
    idx <- ffwhich(df, (Election_Jurisdiction == start.df$state.ab[ii] &
                     Election_Year > start.df$start.year[ii] &
                     Election_Year <= start.df$stop.year[ii]) &
                     (grepl("house", Office_Sought, ignore.case = T) |
                        grepl("assembly", Office_Sought, ignore.case = T)))
  } else {
    idx <- ffwhich(df, (Election_Jurisdiction == start.df$state.ab[ii] &
                          Election_Year > start.df$start.year[ii] &
                          Election_Year <= start.df$stop.year[ii]) &
                     grepl("senate", Office_Sought, ignore.case = T))
  }

  if(length(idx)!=0){
    store.df <-df[idx,]

    write.csv(store.df, paste("StateData/", start.df$state.ab[ii], "-", start.df$start.year[ii], "-",
                              start.df$stop.year[ii], "-", start.df$leg[ii],
                              ".csv", sep=""), row.names=F)
  }

}
